{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/optimization.py:87: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import bert\n",
    "from bert import run_classifier\n",
    "from bert import optimization\n",
    "from bert import tokenization\n",
    "from bert import modeling\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from prepro_utils import preprocess_text, encode_ids, encode_pieces\n",
    "import sentencepiece as spm\n",
    "\n",
    "sp_model = spm.SentencePieceProcessor()\n",
    "sp_model.Load('tiny-bert-v1/sp10m.cased.bert.model')\n",
    "\n",
    "with open('tiny-bert-v1/sp10m.cased.bert.vocab') as fopen:\n",
    "    v = fopen.read().split('\\n')[:-1]\n",
    "v = [i.split('\\t') for i in v]\n",
    "v = {i[0]: i[1] for i in v}\n",
    "\n",
    "class Tokenizer:\n",
    "    def __init__(self, v):\n",
    "        self.vocab = v\n",
    "        pass\n",
    "    \n",
    "    def tokenize(self, string):\n",
    "        return encode_pieces(sp_model, string, return_unicode=False, sample=False)\n",
    "    \n",
    "    def convert_tokens_to_ids(self, tokens):\n",
    "        return [sp_model.PieceToId(piece) for piece in tokens]\n",
    "    \n",
    "    def convert_ids_to_tokens(self, ids):\n",
    "        return [sp_model.IdToPiece(i) for i in ids]\n",
    "    \n",
    "tokenizer = Tokenizer(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "import json\n",
    "\n",
    "left, right, label = [], [], []\n",
    "for file in glob('../text-similarity/*k.json'):\n",
    "    with open(file) as fopen:\n",
    "        x = json.load(fopen)\n",
    "    for i in x:\n",
    "        splitted = i[0].split(' <> ')\n",
    "        if len(splitted) != 2:\n",
    "            continue\n",
    "        left.append(splitted[0])\n",
    "        right.append(splitted[1])\n",
    "        label.append(i[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Bagaimanakah saya boleh menjadi ahli geologi yang baik?',\n",
       "  'Apa yang perlu saya lakukan untuk menjadi ahli geologi yang hebat?',\n",
       "  1),\n",
       " ('Sekiranya saya membeli tiago?',\n",
       "  'Apa yang menjaga kanak-kanak aktif dan jauh dari telefon dan permainan video?',\n",
       "  0),\n",
       " ('Motorola (syarikat): Bolehkah saya menggodam Piagam Motorolla DCX3400 saya?',\n",
       "  'Bagaimana saya mengesan Motorola DCX3400 untuk internet percuma?',\n",
       "  0),\n",
       " ('Apakah kisah Kohinoor (Koh-i-Noor) Diamond?',\n",
       "  'Apa yang akan berlaku jika kerajaan India mencuri kembali berlian Kohinoor (Koh-i-Noor)?',\n",
       "  0),\n",
       " ('Apakah panduan langkah demi langkah untuk melabur dalam pasaran saham di india?',\n",
       "  'Apakah panduan langkah demi langkah untuk melabur dalam pasaran saham?',\n",
       "  0),\n",
       " ('Bilakah anda menggunakan シ bukan し?',\n",
       "  'Bilakah anda menggunakan \"&\" bukan \"dan\"?',\n",
       "  0),\n",
       " ('Astrologi: Saya Capricorn Sun Cap moon dan cap naik ... apa kata itu tentang saya?',\n",
       "  'Saya Capricorn tiga kali ganda (Sun, Moon dan naik di Capricorn) Apa kata ini tentang saya?',\n",
       "  1),\n",
       " ('Yang manakah larut dalam air quikly gula, garam, metana dan karbon di oksida?',\n",
       "  'Ikan mana yang akan bertahan dalam air garam?',\n",
       "  0),\n",
       " ('Kenapa saya mental sangat kesepian? Bagaimana saya boleh menyelesaikannya?',\n",
       "  'Cari sisanya apabila [math] 23 ^ {24} [/ math] dibahagikan dengan 24,23?',\n",
       "  0),\n",
       " ('Bagaimanakah saya dapat meningkatkan kelajuan sambungan internet saya semasa menggunakan VPN?',\n",
       "  'Bagaimana kelajuan Internet dapat ditingkatkan dengan meretas melalui DNS?',\n",
       "  0)]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(zip(left[:10], right[:10], label[:10]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "l = {'contradiction': 0, 'entailment': 1}\n",
    "\n",
    "\n",
    "snli = glob('../text-similarity/part*.json')\n",
    "for file in snli:\n",
    "    with open(file) as fopen:\n",
    "        x = json.load(fopen)\n",
    "    for i in x:\n",
    "        splitted = i[1].split(' <> ')\n",
    "        if len(splitted) != 2:\n",
    "            continue\n",
    "        if i[0] not in l:\n",
    "            continue\n",
    "        left.append(splitted[0])\n",
    "        right.append(splitted[1])\n",
    "        try:\n",
    "            label.append(l[i[0]])\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            print(splitted, i[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Seorang wanita tidur di bawah selimut biru dan putih di dalam kereta.',\n",
       "  'Seorang wanita tidur di dalam kereta.',\n",
       "  1),\n",
       " ('Sesetengah orang di dalam air dan seorang yang berdiri di atas papan.',\n",
       "  'Orang-orang di pantai.',\n",
       "  1),\n",
       " ('Sesetengah orang di dalam air dan seorang yang berdiri di atas papan.',\n",
       "  'Orang ramai sedang bersiap untuk tidur.',\n",
       "  0),\n",
       " ('Lelaki dalam pakaian polis bersebelahan seorang lelaki dalam kostum zombie.',\n",
       "  'Wanita dalam pakaian polis bersebelahan dengan seorang lelaki dalam kostum zombie.',\n",
       "  0),\n",
       " ('Lelaki dalam pakaian polis bersebelahan seorang lelaki dalam kostum zombie.',\n",
       "  'Seseorang dalam pakaian polis bersebelahan seorang lelaki dalam kostum zombie.',\n",
       "  1),\n",
       " ('Seorang lelaki dalam pakaian seragam berdiri di sebelah seorang lelaki dalam kostum seram.',\n",
       "  'Lelaki duduk di sebelah wanita.',\n",
       "  0),\n",
       " ('Di sebalik dermaga yang panjang ke dalam tasik, beberapa orang berenang dan menikmati air.',\n",
       "  'Cuaca sangat buruk di luar.',\n",
       "  0),\n",
       " ('Seorang wanita tidur di bawah selimut biru dan putih di dalam kereta.',\n",
       "  'Seorang wanita memandu di dalam kereta ke arah bandar.',\n",
       "  0),\n",
       " ('Seorang lelaki dalam pakaian seragam berdiri di sebelah seorang lelaki dalam kostum seram.',\n",
       "  'Dua lelaki dalam pakaian berdiri bersebelahan.',\n",
       "  1),\n",
       " ('Di sebalik dermaga yang panjang ke dalam tasik, beberapa orang berenang dan menikmati air.',\n",
       "  'Ada orang di dalam air.',\n",
       "  1)]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(zip(left[-10:], right[-10:], label[-10:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../text-similarity/translated-1.json',\n",
       " '../text-similarity/translated-4.json',\n",
       " '../text-similarity/translated-3.json',\n",
       " '../text-similarity/translated-0.json',\n",
       " '../text-similarity/translated-2.json']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnli = glob('../text-similarity/translated-*.json')\n",
    "mnli"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "for file in mnli:\n",
    "    with open(file) as fopen:\n",
    "        x = json.load(fopen)\n",
    "    for i in x:\n",
    "        if len(i) != 3:\n",
    "            continue\n",
    "        splitted = i[2].split(' <> ')\n",
    "        if len(splitted) != 3:\n",
    "            continue\n",
    "        if i[1] not in l:\n",
    "            continue\n",
    "        left.append(splitted[0])\n",
    "        right.append(splitted[1])\n",
    "        try:\n",
    "            label.append(l[i[1]])\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            print(splitted, i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Vista yang mengagumkan ini pada awalnya dirancang untuk dilihat oleh Napo Leon dari bilik tidurnya di Louvre, yang pada masa itu adalah istana.',\n",
       "  'Pemandangan monumental dibina untuk dilihat oleh Napoleon dari bilik tidurnya di Louvre.',\n",
       "  1),\n",
       " ('Ia menggeram dan tersentak di udara, dan walaupun berjaya menghentak satu korban (yang, dengan sentuhan yang bagus, melekat di bahagian bawah kakinya seperti sepotong permen karet), secara amnya ia bergerak dengan gaya berjalan yang berat sehingga kita mungkin juga kembali pada tahun 60-an menyaksikan Lembah Gwangi. Di manakah kelajuan maut pemangsa?',\n",
       "  'Pemangsa tidak pernah dapat membunuh mangsa.',\n",
       "  0),\n",
       " ('Republik Turki moden bermula pada tahun 1923, tetapi sejarah tanah di dalam sempadannya bermula sejak awal umat manusia.',\n",
       "  'Turki mempunyai sejarah panjang sejak ribuan tahun.',\n",
       "  0),\n",
       " ('Kawasan membeli-belah utama adalah Tsim Sha Tsui di Kowloon, terutama di sepanjang Jalan Nathan; Pusat di Pulau Hong Kong, terutama untuk barang berjenama kelas atas; Causeway Bay dengan harga yang sedikit lebih baik; dan kawasan Hollywood Road.',\n",
       "  'Harga lebih teruk di Causeway Bay.',\n",
       "  0),\n",
       " ('LSC meminta dana untuk melakukan kajian keperluan undang-undang nasional yang baru dalam permintaan anggaran tahun 2004 kepada Kongres; namun, tidak ada dana yang diperuntukkan untuk tujuan tersebut.',\n",
       "  'Kongres tidak memperuntukkan dana untuk semua yang diminta LSC.',\n",
       "  1),\n",
       " ('Cahaya itu kelihatan seperti biasa, di mana langit masih utuh.',\n",
       "  'Cahaya itu hampir seperti biasa.',\n",
       "  1),\n",
       " ('baik pada skala satu hingga sepuluh uh menjadi sepuluh tidak ada jenis undang-undang dan tidak ada uh larangan total saya mungkin akan lebih cenderung ke arah enam atau tujuh um saya rasa seperti larangan total senjata hanya akan meletakkan senjata di tangan penjenayah',\n",
       "  'Saya tidak semestinya bersetuju dengan larangan senjata api.',\n",
       "  1),\n",
       " ('Dua ahli gusti besar, simbolik Malla pertama, orang kuat, mengapit beranda terbuka struktur ini, dan Garuda berlapis emas berlutut di tiang di atas.',\n",
       "  'Garuda batu yang berlutut pada tiang di atas.',\n",
       "  0),\n",
       " ('Dia memegang jawatan itu sejenak, lelaki itu meraung lembut, dan kemudian kembali berdiri.',\n",
       "  'Dia kembali berdiri setelah memegang kedudukannya.',\n",
       "  1),\n",
       " ('Seluruh dunia lagi bimbang dengan konfrontasi nuklear antara kedua-dua negara.',\n",
       "  'Warga planet ini resah tentang kemungkinan perang nuklear.',\n",
       "  1)]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(zip(left[-10:], right[-10:], label[-10:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "BERT_INIT_CHKPNT = 'tiny-bert-v1/model.ckpt'\n",
    "BERT_CONFIG = 'tiny-bert-v1/config.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "MAX_SEQ_LENGTH = 300\n",
    "\n",
    "def _truncate_seq_pair(tokens_a, tokens_b, max_length):\n",
    "    while True:\n",
    "        total_length = len(tokens_a) + len(tokens_b)\n",
    "        if total_length <= max_length:\n",
    "            break\n",
    "        if len(tokens_a) > len(tokens_b):\n",
    "              tokens_a.pop()\n",
    "        else:\n",
    "              tokens_b.pop()\n",
    "                \n",
    "def get_inputs(left, right):\n",
    "\n",
    "    input_ids, input_masks, segment_ids = [], [], []\n",
    "    for i in tqdm(range(len(left))):\n",
    "        tokens_a = tokenizer.tokenize(left[i])\n",
    "        tokens_b = tokenizer.tokenize(right[i])\n",
    "        _truncate_seq_pair(tokens_a, tokens_b, MAX_SEQ_LENGTH - 3)\n",
    "\n",
    "        tokens = []\n",
    "        segment_id = []\n",
    "        tokens.append(\"[CLS]\")\n",
    "        segment_id.append(0)\n",
    "        for token in tokens_a:\n",
    "            tokens.append(token)\n",
    "            segment_id.append(0)\n",
    "        tokens.append(\"[SEP]\")\n",
    "        segment_id.append(0)\n",
    "        for token in tokens_b:\n",
    "            tokens.append(token)\n",
    "            segment_id.append(1)\n",
    "        tokens.append(\"[SEP]\")\n",
    "        segment_id.append(1)\n",
    "        input_id = tokenizer.convert_tokens_to_ids(tokens)\n",
    "        input_mask = [1] * len(input_id)\n",
    "        while len(input_id) < MAX_SEQ_LENGTH:\n",
    "            input_id.append(0)\n",
    "            input_mask.append(0)\n",
    "            segment_id.append(0)\n",
    "\n",
    "        input_ids.append(input_id)\n",
    "        input_masks.append(input_mask)\n",
    "        segment_ids.append(segment_id)\n",
    "    \n",
    "    return input_ids, input_masks, segment_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1041526, 1041526)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(left), len(right)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1041526/1041526 [04:09<00:00, 4182.64it/s]\n"
     ]
    }
   ],
   "source": [
    "input_ids, input_masks, segment_ids = get_inputs(left, right)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:93: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "bert_config = modeling.BertConfig.from_json_file(BERT_CONFIG)\n",
    "epoch = 20\n",
    "batch_size = 128\n",
    "warmup_proportion = 0.1\n",
    "num_train_steps = int(len(left) / batch_size * epoch)\n",
    "num_warmup_steps = int(num_train_steps * warmup_proportion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(\n",
    "        self,\n",
    "        dimension_output,\n",
    "        learning_rate = 2e-5,\n",
    "        training = True\n",
    "    ):\n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.segment_ids = tf.placeholder(tf.int32, [None, None])\n",
    "        self.input_masks = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None])\n",
    "        \n",
    "        model = modeling.BertModel(\n",
    "            config=bert_config,\n",
    "            is_training=training,\n",
    "            input_ids=self.X,\n",
    "            input_mask=self.input_masks,\n",
    "            token_type_ids=self.segment_ids,\n",
    "            use_one_hot_embeddings=False)\n",
    "        \n",
    "        output_layer = model.get_pooled_output()\n",
    "        self.logits = tf.layers.dense(output_layer, dimension_output)\n",
    "        self.logits = tf.identity(self.logits, name = 'logits')\n",
    "        \n",
    "        self.cost = tf.reduce_mean(\n",
    "            tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
    "                logits = self.logits, labels = self.Y\n",
    "            )\n",
    "        )\n",
    "        \n",
    "        self.optimizer = optimization.create_optimizer(self.cost, learning_rate, \n",
    "                                                       num_train_steps, num_warmup_steps, False)\n",
    "        correct_pred = tf.equal(\n",
    "            tf.argmax(self.logits, 1, output_type = tf.int32), self.Y\n",
    "        )\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:171: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:409: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:490: The name tf.assert_less_equal is deprecated. Please use tf.compat.v1.assert_less_equal instead.\n",
      "\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:358: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:671: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/optimization.py:27: The name tf.train.get_or_create_global_step is deprecated. Please use tf.compat.v1.train.get_or_create_global_step instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/optimization.py:32: The name tf.train.polynomial_decay is deprecated. Please use tf.compat.v1.train.polynomial_decay instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/keras/optimizer_v2/learning_rate_schedule.py:409: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/optimization.py:70: The name tf.trainable_variables is deprecated. Please use tf.compat.v1.trainable_variables instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:1205: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/training/saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "INFO:tensorflow:Restoring parameters from tiny-bert-v1/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "dimension_output = 2\n",
    "learning_rate = 2e-5\n",
    "\n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(\n",
    "    dimension_output,\n",
    "    learning_rate\n",
    ")\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "var_lists = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope = 'bert')\n",
    "saver = tf.train.Saver(var_list = var_lists)\n",
    "saver.restore(sess, BERT_INIT_CHKPNT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_input_ids, test_input_ids, train_input_masks, test_input_masks, train_segment_ids, test_segment_ids, train_Y, test_Y = train_test_split(\n",
    "    input_ids, input_masks, segment_ids, label, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 6510/6510 [40:38<00:00,  2.67it/s, accuracy=0.765, cost=0.496]\n",
      "test minibatch loop: 100%|██████████| 1628/1628 [04:34<00:00,  5.93it/s, accuracy=0.78, cost=0.456] \n",
      "train minibatch loop:   0%|          | 0/6510 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.000000, current acc: 0.785645\n",
      "time taken: 2713.207044363022\n",
      "epoch: 0, training loss: 0.524101, training acc: 0.725539, valid loss: 0.445311, valid acc: 0.785645\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 6510/6510 [40:38<00:00,  2.67it/s, accuracy=0.868, cost=0.341]\n",
      "test minibatch loop: 100%|██████████| 1628/1628 [04:34<00:00,  5.92it/s, accuracy=0.86, cost=0.347] \n",
      "train minibatch loop:   0%|          | 0/6510 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, pass acc: 0.785645, current acc: 0.825923\n",
      "time taken: 2713.2396500110626\n",
      "epoch: 1, training loss: 0.405105, training acc: 0.810648, valid loss: 0.380279, valid acc: 0.825923\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 6510/6510 [40:39<00:00,  2.67it/s, accuracy=0.794, cost=0.323]\n",
      "test minibatch loop: 100%|██████████| 1628/1628 [04:35<00:00,  5.92it/s, accuracy=0.9, cost=0.253]  \n",
      "train minibatch loop:   0%|          | 0/6510 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 2, pass acc: 0.825923, current acc: 0.842690\n",
      "time taken: 2714.2310569286346\n",
      "epoch: 2, training loss: 0.351967, training acc: 0.841187, valid loss: 0.350611, valid acc: 0.842690\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 6510/6510 [40:39<00:00,  2.67it/s, accuracy=0.853, cost=0.24] \n",
      "test minibatch loop: 100%|██████████| 1628/1628 [04:34<00:00,  5.92it/s, accuracy=0.92, cost=0.253] \n",
      "train minibatch loop:   0%|          | 0/6510 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 3, pass acc: 0.842690, current acc: 0.852814\n",
      "time taken: 2714.708259344101\n",
      "epoch: 3, training loss: 0.314684, training acc: 0.861928, valid loss: 0.334570, valid acc: 0.852814\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 6510/6510 [40:39<00:00,  2.67it/s, accuracy=0.912, cost=0.221]\n",
      "test minibatch loop: 100%|██████████| 1628/1628 [04:34<00:00,  5.93it/s, accuracy=0.86, cost=0.384] \n",
      "train minibatch loop:   0%|          | 0/6510 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 4, pass acc: 0.852814, current acc: 0.857441\n",
      "time taken: 2714.647777080536\n",
      "epoch: 4, training loss: 0.288701, training acc: 0.875240, valid loss: 0.327252, valid acc: 0.857441\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 6510/6510 [40:37<00:00,  2.67it/s, accuracy=0.897, cost=0.201]\n",
      "test minibatch loop: 100%|██████████| 1628/1628 [04:33<00:00,  5.94it/s, accuracy=0.9, cost=0.277]  \n",
      "train minibatch loop:   0%|          | 0/6510 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 5, pass acc: 0.857441, current acc: 0.861463\n",
      "time taken: 2711.6535017490387\n",
      "epoch: 5, training loss: 0.267652, training acc: 0.886356, valid loss: 0.321805, valid acc: 0.861463\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 6510/6510 [40:34<00:00,  2.67it/s, accuracy=0.941, cost=0.146] \n",
      "test minibatch loop: 100%|██████████| 1628/1628 [04:34<00:00,  5.94it/s, accuracy=0.9, cost=0.271]  \n",
      "train minibatch loop:   0%|          | 0/6510 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 6, pass acc: 0.861463, current acc: 0.863426\n",
      "time taken: 2708.7250673770905\n",
      "epoch: 6, training loss: 0.249877, training acc: 0.895392, valid loss: 0.329341, valid acc: 0.863426\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 6510/6510 [40:34<00:00,  2.67it/s, accuracy=0.941, cost=0.157] \n",
      "test minibatch loop: 100%|██████████| 1628/1628 [04:33<00:00,  5.95it/s, accuracy=0.88, cost=0.295] \n",
      "train minibatch loop:   0%|          | 0/6510 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 7, pass acc: 0.863426, current acc: 0.864474\n",
      "time taken: 2708.3127732276917\n",
      "epoch: 7, training loss: 0.234747, training acc: 0.902913, valid loss: 0.329308, valid acc: 0.864474\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 6510/6510 [40:33<00:00,  2.67it/s, accuracy=0.926, cost=0.17]  \n",
      "test minibatch loop: 100%|██████████| 1628/1628 [04:33<00:00,  5.95it/s, accuracy=0.86, cost=0.297] \n",
      "train minibatch loop:   0%|          | 0/6510 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 8, pass acc: 0.864474, current acc: 0.865158\n",
      "time taken: 2707.6398663520813\n",
      "epoch: 8, training loss: 0.221076, training acc: 0.909245, valid loss: 0.331831, valid acc: 0.865158\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 6510/6510 [40:33<00:00,  2.67it/s, accuracy=0.956, cost=0.11]  \n",
      "test minibatch loop: 100%|██████████| 1628/1628 [04:33<00:00,  5.95it/s, accuracy=0.9, cost=0.35]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 2707.5254366397858\n",
      "epoch: 9, training loss: 0.208686, training acc: 0.915261, valid loss: 0.345386, valid acc: 0.864755\n",
      "\n",
      "break epoch:10\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import time\n",
    "\n",
    "EARLY_STOPPING, CURRENT_CHECKPOINT, CURRENT_ACC, EPOCH = 1, 0, 0, 0\n",
    "\n",
    "while True:\n",
    "    lasttime = time.time()\n",
    "    if CURRENT_CHECKPOINT == EARLY_STOPPING:\n",
    "        print('break epoch:%d\\n' % (EPOCH))\n",
    "        break\n",
    "\n",
    "    train_acc, train_loss, test_acc, test_loss = [], [], [], []\n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_input_ids), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_input_ids))\n",
    "        batch_x = train_input_ids[i: index]\n",
    "        batch_masks = train_input_masks[i: index]\n",
    "        batch_segment = train_segment_ids[i: index]\n",
    "        batch_y = train_Y[i: index]\n",
    "        acc, cost, _ = sess.run(\n",
    "            [model.accuracy, model.cost, model.optimizer],\n",
    "            feed_dict = {\n",
    "                model.Y: batch_y,\n",
    "                model.X: batch_x,\n",
    "                model.segment_ids: batch_segment,\n",
    "                model.input_masks: batch_masks\n",
    "            },\n",
    "        )\n",
    "        assert not np.isnan(cost)\n",
    "        train_loss.append(cost)\n",
    "        train_acc.append(acc)\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc)\n",
    "    pbar = tqdm(range(0, len(test_input_ids), batch_size), desc = 'test minibatch loop')\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_input_ids))\n",
    "        batch_x = test_input_ids[i: index]\n",
    "        batch_masks = test_input_masks[i: index]\n",
    "        batch_segment = test_segment_ids[i: index]\n",
    "        batch_y = test_Y[i: index]\n",
    "        acc, cost = sess.run(\n",
    "            [model.accuracy, model.cost],\n",
    "            feed_dict = {\n",
    "                model.Y: batch_y,\n",
    "                model.X: batch_x,\n",
    "                model.segment_ids: batch_segment,\n",
    "                model.input_masks: batch_masks\n",
    "            },\n",
    "        )\n",
    "        test_loss.append(cost)\n",
    "        test_acc.append(acc)\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc)\n",
    "\n",
    "    train_loss = np.mean(train_loss)\n",
    "    train_acc = np.mean(train_acc)\n",
    "    test_loss = np.mean(test_loss)\n",
    "    test_acc = np.mean(test_acc)\n",
    "\n",
    "    if test_acc > CURRENT_ACC:\n",
    "        print(\n",
    "            'epoch: %d, pass acc: %f, current acc: %f'\n",
    "            % (EPOCH, CURRENT_ACC, test_acc)\n",
    "        )\n",
    "        CURRENT_ACC = test_acc\n",
    "        CURRENT_CHECKPOINT = 0\n",
    "    else:\n",
    "        CURRENT_CHECKPOINT += 1\n",
    "    print('time taken:', time.time() - lasttime)\n",
    "    print(\n",
    "        'epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'\n",
    "        % (EPOCH, train_loss, train_acc, test_loss, test_acc)\n",
    "    )\n",
    "    EPOCH += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'tiny-bert-similarity/model.ckpt'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'tiny-bert-similarity/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/client/session.py:1735: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from tiny-bert-similarity/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "dimension_output = 2\n",
    "learning_rate = 2e-5\n",
    "\n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(\n",
    "    dimension_output,\n",
    "    learning_rate,\n",
    "    training = False\n",
    ")\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.restore(sess, 'tiny-bert-similarity/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Placeholder',\n",
       " 'Placeholder_1',\n",
       " 'Placeholder_2',\n",
       " 'Placeholder_3',\n",
       " 'bert/embeddings/word_embeddings',\n",
       " 'bert/embeddings/token_type_embeddings',\n",
       " 'bert/embeddings/position_embeddings',\n",
       " 'bert/embeddings/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_0/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/query/bias',\n",
       " 'bert/encoder/layer_0/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/key/bias',\n",
       " 'bert/encoder/layer_0/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/value/bias',\n",
       " 'bert/encoder/layer_0/attention/self/Softmax',\n",
       " 'bert/encoder/layer_0/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_0/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_0/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_0/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_0/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_0/output/dense/kernel',\n",
       " 'bert/encoder/layer_0/output/dense/bias',\n",
       " 'bert/encoder/layer_0/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_1/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/query/bias',\n",
       " 'bert/encoder/layer_1/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/key/bias',\n",
       " 'bert/encoder/layer_1/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/value/bias',\n",
       " 'bert/encoder/layer_1/attention/self/Softmax',\n",
       " 'bert/encoder/layer_1/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_1/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_1/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_1/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_1/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_1/output/dense/kernel',\n",
       " 'bert/encoder/layer_1/output/dense/bias',\n",
       " 'bert/encoder/layer_1/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_2/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_2/attention/self/query/bias',\n",
       " 'bert/encoder/layer_2/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_2/attention/self/key/bias',\n",
       " 'bert/encoder/layer_2/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_2/attention/self/value/bias',\n",
       " 'bert/encoder/layer_2/attention/self/Softmax',\n",
       " 'bert/encoder/layer_2/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_2/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_2/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_2/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_2/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_2/output/dense/kernel',\n",
       " 'bert/encoder/layer_2/output/dense/bias',\n",
       " 'bert/encoder/layer_2/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_3/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_3/attention/self/query/bias',\n",
       " 'bert/encoder/layer_3/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_3/attention/self/key/bias',\n",
       " 'bert/encoder/layer_3/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_3/attention/self/value/bias',\n",
       " 'bert/encoder/layer_3/attention/self/Softmax',\n",
       " 'bert/encoder/layer_3/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_3/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_3/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_3/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_3/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_3/output/dense/kernel',\n",
       " 'bert/encoder/layer_3/output/dense/bias',\n",
       " 'bert/encoder/layer_3/output/LayerNorm/gamma',\n",
       " 'bert/pooler/dense/kernel',\n",
       " 'bert/pooler/dense/bias',\n",
       " 'dense/kernel',\n",
       " 'dense/bias',\n",
       " 'logits',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/mul_1']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or 'logits' in n.name\n",
    "        or 'alphas' in n.name\n",
    "        or 'self/Softmax' in n.name)\n",
    "        and 'adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'global_step' not in n.name\n",
    "    ]\n",
    ")\n",
    "strings.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "validation minibatch loop: 100%|██████████| 1628/1628 [03:18<00:00,  8.21it/s]\n"
     ]
    }
   ],
   "source": [
    "real_Y, predict_Y = [], []\n",
    "\n",
    "pbar = tqdm(\n",
    "    range(0, len(test_input_ids), batch_size), desc = 'validation minibatch loop'\n",
    ")\n",
    "for i in pbar:\n",
    "    index = min(i + batch_size, len(test_input_ids))\n",
    "    batch_x = test_input_ids[i: index]\n",
    "    batch_masks = test_input_masks[i: index]\n",
    "    batch_segment = test_segment_ids[i: index]\n",
    "    batch_y = test_Y[i: index]\n",
    "    predict_Y += np.argmax(sess.run(model.logits,\n",
    "            feed_dict = {\n",
    "                model.Y: batch_y,\n",
    "                model.X: batch_x,\n",
    "                model.segment_ids: batch_segment,\n",
    "                model.input_masks: batch_masks\n",
    "            },\n",
    "    ), 1, ).tolist()\n",
    "    real_Y += batch_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      " not similar    0.90845   0.85704   0.88200    114843\n",
      "     similar    0.83576   0.89387   0.86384     93463\n",
      "\n",
      "    accuracy                        0.87357    208306\n",
      "   macro avg    0.87210   0.87546   0.87292    208306\n",
      "weighted avg    0.87583   0.87357   0.87385    208306\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "\n",
    "print(\n",
    "    metrics.classification_report(\n",
    "        real_Y, predict_Y, target_names = ['not similar', 'similar'],\n",
    "        digits = 5\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:00<00:00, 1290.56it/s]\n"
     ]
    }
   ],
   "source": [
    "test_ids, test_masks, test_segment = get_inputs(['rakyat mahu tukar kerajaan'], \n",
    "                                                 ['teks ini adalah mengenai najib'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 3.0846386, -3.0090528]], dtype=float32)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run(\n",
    "    model.logits,\n",
    "    feed_dict = {\n",
    "        model.X: test_ids,\n",
    "        model.segment_ids: test_segment,\n",
    "        model.input_masks: test_masks\n",
    "    },\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(','),\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from tiny-bert-similarity/model.ckpt\n",
      "WARNING:tensorflow:From <ipython-input-35-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/graph_util_impl.py:270: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n",
      "INFO:tensorflow:Froze 73 variables.\n",
      "INFO:tensorflow:Converted 73 variables to const ops.\n",
      "2385 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('tiny-bert-similarity', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "\n",
    "bucketName = 'huseinhouse-storage'\n",
    "Key = 'tiny-bert-similarity/frozen_model.pb'\n",
    "outPutname = \"v36/similarity/tiny-bert-similarity.pb\"\n",
    "\n",
    "s3 = boto3.client('s3')\n",
    "\n",
    "s3.upload_file(Key,bucketName,outPutname)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
